# -*- coding: utf-8 -*-

# !/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author;Tsukasa

import json
from multiprocessing import Pool
import requests
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
import re
import pandas as pd
import pymongo
import os
import sys
import traceback

ua = UserAgent()
headers1 = {'User-Agent': 'ua.ramdom'}
Mongo_Url = 'localhost'
Mongo_DB = 'Lianjia'
Mongo_TABLE = 'ershoufang'
t_sub_area = 'sub_area'
client = pymongo.MongoClient(Mongo_Url)
db = client[Mongo_DB]
base_url = "https://bj.lianjia.com"


def get_page_of_ershoufang(sub_area_url):  # 分析url解析出每一页的详细url
    print(sub_area_url)
    res = requests.get(base_url + sub_area_url, 'lxml', headers=headers1)
    total_pages = 1
    if res.status_code == 200:
        soup = BeautifulSoup(res.text, 'lxml')
        page_box = soup.select_one('.house-lst-page-box')
        if page_box is not None and page_box.has_attr('page-data'):
            page_data = page_box['page-data']
            total_pages = page_data[13:].split(',')[0]
    ershoufang_id_list = []
    # print("总页数：" + total_pages)
    for page in range(1, int(total_pages) + 1):
        res = requests.get(base_url + sub_area_url + 'pg' + str(page), 'lxml', headers=headers1)
        if res.status_code == 200:
            re_set = re.compile('<li.*?class="clear.*?<a.*?class="img.*?".*?data-housecode="(.*?)"')
            re_get = re.findall(re_set, res.text)
            ershoufang_id_list += re_get

    return ershoufang_id_list


def update_to_MongoDB(ershoufang_id):
    print(ershoufang_id)
    db[Mongo_TABLE].update({'链家编号': ershoufang_id}, {'$set': {"链家编号": ershoufang_id}})


if __name__ == '__main__':

    with Pool(os.cpu_count()) as pool:
        for sub_area_info in db[t_sub_area].find().batch_size(50):
            pool.map(update_to_MongoDB,
                [ershoufang_id for ershoufang_id in get_page_of_ershoufang(sub_area_info['sub_area_url'])])
